#!/usr/bin/env python
# -*- coding:utf-8 -*-
# @FileName  :huaban_boards.py
# @Time      :2023/7/27 
# @Author    :CL
# @email     :1037654919@qq.com
#
import os
import time
import random
from tqdm import tqdm
from utils import MongoDBUtil, get_html, mongo_manager
from multiprocessing import Pool
from huaban_util import Huaban

MongoDBUtil = MongoDBUtil()
images_keywords = mongo_manager('images_keywords',db ='car_images')



def insert_data(tablename, datas):
    if len(datas) > 0 and datas is not None:
        for data in datas:
            # data['_id'] = data['id']
            try:
                # print(data)
                MongoDBUtil.insert_one(tablename, data)
            except Exception as e:
                print(e)
    else:
        pass


def get_boards(seed):

    keyword = seed['keyword']
    keyword_en = seed['keyword_en']

    response = huaban.get_board_id(keyword, 1)
    board_count = response['board_count']
    pages = int(board_count) / 20
    print(keyword, ':', pages, '页')
    for page in range(1, int(pages) + 1):
        response = huaban.get_board_id(keyword, page)
        datas = response['boards']
        print('len(datas):', len(datas))
        for data in datas:
            result = {}
            result['_id'] = data['board_id']
            result['board_id'] = data['board_id']
            result['keyword'] = keyword
            result['keyword_e'] = keyword_en
            result['page'] = page
            result['user_id'] = data['user_id']
            result['title'] = data['title']
            result['pin_count'] = data['pin_count']
            result['seq'] = data['seq']
            try:
                # print(data)
                MongoDBUtil.insert_one('huaban_boards', result)
            except Exception as e:
                print(e)
            # break
        if len(datas) < 20:
            break
    seed['huaban'] = 'done'
    images_keywords.updateOne({'_id': seed['_id']}, seed)
    return


if __name__ == "__main__":
    huaban = Huaban()

    seeds = images_keywords.findAll({'huaban':None,'_for':'person'})
    lists =[]
    for seed in seeds:
        lists.append(seed)
    pool = Pool(processes=30)
    pool.map(get_boards, lists)
    pool.close()  # 关闭进程池，不再接受新的进程
    pool.join()

    images_keywords.close()

